# -*- coding: utf-8 -*-
import scrapy
from scrapy.http import Request
from urllib.parse import urljoin
from items import ershou_lianjiaItemLoader
from items import ershou_lianjiaItem
from selenium import webdriver


class ErshouLianjiaSpider(scrapy.Spider):
    name = 'ershou_lianjia'
    allowed_domains = ['bj.lianjia.com/ershoufang/']
    start_urls = ['https://bj.lianjia.com/ershoufang/']

    def __init__(self):
        firefox_profile = webdriver.FirefoxProfile()
        firefox_profile.set_preference('permissions.default.image', 2)

        self.driver = webdriver.Firefox(firefox_profile=firefox_profile,
                                        executable_path="/home/demeen/Desktop/source/geckodriver")
        super(ErshouLianjiaSpider, self).__init__()
        from scrapy.xlib.pydispatch import dispatcher
        from scrapy import signals
        dispatcher.connect(self.spider_closed, signals.spider_closed)

    def spider_closed(self, spider):
        print("spider close")
        self.driver.quit()

    def parse(self, response):
        house_list = response.css(".sellListContent li.clear .title a::attr(href)").extract()
        for house in house_list:
            yield Request(url=house, callback=self.parse_house, dont_filter=True)
        next_page = response.css(".contentBottom .house-lst-page-box a:last-child")
        next_page = next_page.css("a::attr(href)").extract_first()
        if next_page:
            next_url = urljoin(response.url, next_page)
            # todo dont_filter=True
            yield Request(url=next_url, callback=self.parse)

    def parse_house(self, response):
        item_loader = ershou_lianjiaItemLoader(item=ershou_lianjiaItem(), response=response)
        item_loader.add_value("id", str(response.url).split('/')[-1])
        title = response.css(".sellDetailHeader .content .main::text").extract_first()
        item_loader.add_value("title", title)
        top_plane = response.css(".overview .content")
        house_price = top_plane.css(".price span::text").extract_first() + top_plane.css(
            ".unit span::text").extract_first()
    
        item_loader.add_value("price", house_price)
        house_unit_price = top_plane.css(".unitPrice span::text").extract_first()
        item_loader.add_value("unit_price", house_unit_price)
        house_tax_pay = top_plane.css("#tax-text span::attr(title)").extract_first()
        item_loader.add_value("tax_pay", house_tax_pay)
        house_room_1 = top_plane.css(".houseInfo .room .mainInfo::text").extract_first()
        item_loader.add_value("room1", house_room_1)
        house_room_2 = top_plane.css(".houseInfo .room .subInfo::text").extract_first()
        item_loader.add_value("room2", house_room_2)
        house_type_1 = top_plane.css(".houseInfo .type .mainInfo::text").extract_first()
        item_loader.add_value("type1", house_type_1)
        house_type_2 = top_plane.css(".houseInfo .type .subInfo::text").extract_first()
        item_loader.add_value("type2", house_type_2)
        house_area_1 = top_plane.css(".houseInfo .area .mainInfo::text").extract_first()
        item_loader.add_value("area1", house_area_1)
        house_area_2 = top_plane.css(".houseInfo .area .subInfo::text").extract_first()
        item_loader.add_value("area2", house_area_2)
        # community
        house_community_name = top_plane.css(".aroundInfo .communityName .info").css("a::text").extract_first()
        item_loader.add_value("community_name", house_community_name)
        house_community_area = top_plane.css(".aroundInfo .areaName .info a").css("span::text").extract()
        house_area = ""
        for area in house_community_area:
            house_area = house_area + area
        if house_area:
            item_loader.add_value("community_area", house_area)
        else:
            item_loader.add_value("community_area", "null")
        # details
        base_plane = response.css("#introduction .introContent li::text").extract()
        item_loader.add_value('detail0', base_plane[0])
        item_loader.add_value('detail1', base_plane[1])
        item_loader.add_value('detail2', base_plane[2])
        item_loader.add_value('detail3', base_plane[3])
        item_loader.add_value('detail4', base_plane[4])
        item_loader.add_value('detail5', base_plane[5])
        item_loader.add_value('detail6', base_plane[6])
        item_loader.add_value('detail7', base_plane[7])
        item_loader.add_value('detail8', base_plane[8])
        item_loader.add_value('detail9', base_plane[9])
        item_loader.add_value('detail10', base_plane[10])
        item_loader.add_value('detail11', base_plane[11])
        item_loader.add_value('detail12', base_plane[12])
        item_loader.add_value('detail13', base_plane[13])
        item_loader.add_value('detail14', base_plane[14])
        item_loader.add_value('detail15', base_plane[15])
        item_loader.add_value('detail16', base_plane[16])
        item_loader.add_value('detail17', base_plane[17])
        item_loader.add_value('detail18', base_plane[18])
        item_loader.add_value('detail19', base_plane[19])
        special = response.css(".introContent.showbasemore")  # can't know length
        house_tag_list = special.css(".tags.clear .content a::text").extract()
        tag = ""
        for house_tag in house_tag_list:
            tag = tag + house_tag + ","
        item_loader.add_value("tag", tag)
        house_other_special = special.css(".baseattribute.clear .content::text").extract()
        # see time
        see_plane = response.css("#record")
        house_see_in_seven = see_plane.css(".panel .count::text").extract_first()
        item_loader.add_value("visit_in_seven", house_see_in_seven)
        house_see_in_thirty = see_plane.css(".panel .totalCount span::text").extract_first()
        item_loader.add_value("visit_in_thirty", house_see_in_thirty)
        sp = ""
        for other_special in house_other_special:
            sp = sp + "@" + str(other_special).strip()
        item_loader.add_value("long_detail", sp)
        item = item_loader.load_item()
        yield item
